<?php
/**
 * run with command
 * php think weblog -f filepath -m pcntl -w 3
 */

namespace addons\weblog\library;

ini_set('display_errors', 'on');

use think\Config;
use think\console\Command;
use think\console\Input;
use think\console\input\Argument;
use think\console\input\Option;
use think\console\Output;
use think\Db;
use Exception;
use think\exception\PDOException;
use think\exception\ValidateException;

class Import extends Command
{
    protected $limit = 500;

    /**
     * 创建命令
     * @author Created by Xing <464401240@qq.com>
     */
    protected function configure()
    {
        $this->setName('weblog')
            ->addOption('file', 'f', Option::VALUE_REQUIRED, 'file path')
            ->addOption('workernum', 'w', Option::VALUE_REQUIRED, 'worker number', '1')
            ->setDescription('Say Hello');
    }

    /**
     * 执行命令
     * @author Created by Xing <464401240@qq.com>
     */
    protected function execute(Input $input, Output $output)
    {
        $file = '';
        if ($this->input->hasOption('file')) {
            $file = $this->input->getOption('file');
        }
        if (!file_exists($file)) {
            $this->output->writeln(
                "<error>The file does not exist or has no permissions.</error>"
            );
            return false;
        }
        $this->pcntl($file);
    }

    /**
     * pcntl多线程导入
     * @author Created by Xing <464401240@qq.com>
     */
    private function pcntl($file)
    {
        $t1 = microtime(true);
        $this->output->writeln('Start import...');
        // 必须加载扩展
        if (!function_exists("pcntl_fork") || !function_exists("pcntl_signal") || !function_exists("posix_getpid")) {
            $this->output->error('pcntl & posix_getpid extention is must.');
            return false;
        }

        //添加或更新站点
        $site_id = $this->site($file);
        if (!$site_id) {
            $this->output->error('Unexpected end.');
            return false;
        }

        //所有的蜘蛛
        $spider = \app\admin\model\weblog\Spider::all();
        foreach ($spider as &$row) {
            $row['keyword'] = strtoupper($row['keyword']);
        }
        unset($row);

        $workernum = 1;
        if ($this->input->hasOption('workernum')) {
            $workernum = intval($this->input->getOption('workernum'));
        }

        $log = new \SplFileObject($file, 'rb');
        $max = 0;
        foreach ($log as $row) {
            $max++;
        }
        pcntl_signal(SIGCHLD, SIG_IGN); //信号处理器-如果父进程不关心子进程什么时候结束,子进程结束后，内核会回收
        $i = 0;
        $pids = [];
        Db::getConnection()->close()->free();
        while ($i < $workernum) {
            $start = ceil($max / $workernum * $i);
            $end   = ceil($max / $workernum * ($i + 1));
            //创建子进程
            $pids[$i]   = pcntl_fork();
            //父进程和子进程都会执行以下代码
            if ($pids[$i] == -1) {
                // 创建子进程错误，返回-1
                //die('could not fork');
            } elseif ($pids[$i]) {
                //父进程会得到子进程号，所以这里是父进程执行的逻辑
                //如果不需要阻塞进程，而又想得到子进程的退出状态，则可以注释掉pcntl_wait($status)语句，或写成：
                //pcntl_wait($status); //等待子进程中断，防止子进程成为僵尸进程
            } else {
                //子进程得到的$pid 为0，所以这里是子进程的逻辑
                $this->import($file, $start, $end, $spider, $site_id);
                exit();
            }
            $i++;
        }

        foreach ($pids as $pid) {
            if ($pid) {
                pcntl_waitpid($pid, $status);
            }
        }

        $this->siteStatistical($site_id);   //统计站点信息
        $this->spiderStatistics($site_id);  //统计蜘蛛信息
        $this->codeStatistics($site_id);    //统计状态码信息
        $this->output->writeln('<info>Data analysis completed. Ok</info>');

        $filesize = $this->getSize($file);
        $t2 = microtime(true);
        $this->output->writeln(
            "<info>File size: {$filesize}\nWorker number: {$workernum}\nRun time: " . round($t2 - $t1, 4) . "s\nThe import is complete.</info>"
        );
    }

    /**
     * 导入log
     * @author Created by Xing <464401240@qq.com>
     */
    private function import($file, $start, $end, $spider, $site_id)
    {
        $pid = posix_getpid();
        $pre = '/^(\d{1,3}.\d{1,3}.\d{1,3}.\d{1,3})\s(.*?)\s(.*?)\s\[(.*?)\]\s"(.*?)\s(.*?)\s(.*?)\"\s(\d{3})\s(\d+|-)\s"(.*?)"\s\"(.*?)\"$/u';
        $data = [];
        $log = new \SplFileObject($file, 'rb');
        $log->seek($start);     // 转到第N行, seek方法参数从0开始计数
        $count = $end - $start; // 25-0=25

        $db = Db::name('weblog_log');
        Db::startTrans();
        try {
            for ($i = 1; $i <= $count; $i++) {
                $line = $log->current();
                if ($line && preg_match($pre, $line, $match) !== false && count($match) == 12) {
                    $data[] = [
                        'ip'         => $match[1],
                        'sign'       => $match[2],
                        'username'   => $match[3],
                        'datetime'   => strtotime($match[4]),
                        'method'     => $match[5],
                        'url'        => $match[6],
                        'url_code'   => md5($match[6]),
                        'protocol'   => $match[7],
                        'code'       => $match[8],
                        'size'       => $match[9],
                        'referer'    => $match[10],
                        //'other_info' => $match[11],
                        'spider_id'  => $this->getSpider($spider, $match[11]),
                        'site_id'    => $site_id,
                        'pid'        => $pid,
                    ];
                    $line = substr($line, 0, 100) . '... <info>ok</info>';
                } else {
                    $line = substr($line, 0, 100) . '... <error>fail</error>';
                }
                if (count($data) == $this->limit || ($i >= $count)) {
                    $db->insertAll($data, false, $this->limit);
                    $data = [];
                }
                $log->next();// 下一行
                $this->output->writeln('<comment>line# </comment>' . $line);
            }
            Db::commit();
        } catch (ValidateException $e) {
            $this->output->error($e->getMessage());
            Db::rollback();
            return false;
        } catch (PDOException $e) {
            $this->output->error($e->getMessage());
            Db::rollback();
            return false;
        } catch (Exception $e) {
            $this->output->error($e->getMessage());
            Db::rollback();
            return false;
        }

        $this->urlStatistics($site_id, $pid);
        $this->ipStatistics($site_id, $pid);
    }

    /**
     * 得到蜘蛛ID
     *
     * @param 类型 $spider 所有的蜘蛛
     * @param 类型 $str 字符串
     * @return int 返回类型
     * @example 示例
     * @author Created by Xing <464401240@qq.com>
     */
    private function getSpider($spider = [], $str = '')
    {
        $id = 0;
        if (empty($spider)) {
            return $id;
        }

        $str = strtoupper($str);
        foreach ($spider as $val) {
            if (strpos($str, $val['keyword']) !== false) {
                $id = $val['id'];
                continue;
            }
        }
        return $id;
    }

    /**
     * GET站点ID
     * @param 类型 $file 文件路径
     * @return int 返回类型
     * @author Created by Xing <464401240@qq.com>
     */
    private function site($file)
    {
        $fileinfo = pathinfo($file);
        //添加或更新站点
        $model = new \app\admin\model\weblog\Site;
        $site_id = $model->where(['log_file' => $file, 'name' => $fileinfo['filename']])->value('id');
        if ($site_id) {
            $this->output->writeln('Deleting site_id=' . $site_id . ' data from the weblog_log table');
            Db::name('weblog_log')->where('site_id', '=', $site_id)->delete(true);
            $this->output->writeln('Deleting site_id=' . $site_id . ' data from the weblog_site_statistics table');
            Db::name('weblog_site_statistics')->where('site_id', '=', $site_id)->delete(true);
            $this->output->writeln('Deleting site_id=' . $site_id . ' data from the weblog_spider_statistics table');
            Db::name('weblog_spider_statistics')->where('site_id', '=', $site_id)->delete(true);
            $this->output->writeln('Deleting site_id=' . $site_id . ' data from the weblog_url_statistics table');
            Db::name('weblog_url_statistics')->where('site_id', '=', $site_id)->delete(true);
            $this->output->writeln('Deleting site_id=' . $site_id . ' data from the weblog_ip_statistics table');
            Db::name('weblog_ip_statistics')->where('site_id', '=', $site_id)->delete(true);
            $this->output->writeln('Deleting site_id=' . $site_id . ' data from the weblog_code_statistics table');
            Db::name('weblog_code_statistics')->where('site_id', '=', $site_id)->delete(true);
        } else {
            $this->output->writeln('Add site data...');
            $model->log_file = $file;
            $model->name = $fileinfo['filename'];
            $model->save();
            $site_id = $model->id;
        }
        return $site_id;
    }

    /**
     * 状态码统计
     * @param 类型 $site_id 站点id
     * @return bool 返回类型
     * @author Created by Xing <464401240@qq.com>
     */
    private function codeStatistics($site_id)
    {
        $this->output->writeln('Start statistical code data...');
        $model = new \app\admin\model\weblog\Log();
        $where['site_id'] = $site_id;
        $data = [];
        $list  = $model->group('code')->where($where)->field('count(code) as visits_count,code')->select();
        foreach ($list as $row) {
            $data[] = [
                'site_id' => $site_id,
                'code' => $row->code,
                'visits_count' => $row->visits_count
            ];
        }

        if (empty($data)) {
            return false;
        }
        $model = new \app\admin\model\weblog\CodeStatistics();
        $model->saveAll($data);
        return true;
    }

    /**
     * 蜘蛛统计
     * @param 类型 $site_id 站点id
     * @return bool 返回类型
     * @author Created by Xing <464401240@qq.com>
     */
    private function spiderStatistics($site_id)
    {
        $this->output->writeln('Start statistical spider data...');
        $spider   = \app\admin\model\weblog\Spider::all();
        $spider[] = ['id' => 0, 'name' => '其他'];

        $data = [];
        $where['site_id'] = $site_id;
        $model = new \app\admin\model\weblog\Log();
        foreach ($spider as $row) {
            $where['spider_id'] = $row['id'];
            $total = $model->where($where)->count('id');
            if (!$total) {
                continue;
            }

            $data[] = [
                'site_id' => $site_id,
                'spider_id' => $row['id'],
                'visits_count' => $total,
            ];
        }

        if (empty($data)) {
            return false;
        }
        $model = new \app\admin\model\weblog\SpiderStatistics();
        $model->saveAll($data);
        return true;
    }

    /**
     * 站点数据统计
     * @param 类型 $site_id 站点id
     * @return bool 返回类型
     * @author Created by Xing <464401240@qq.com>
     */
    private function siteStatistical($site_id)
    {
        $this->output->writeln('Start statistical site data...');
        $where['site_id'] = $site_id;
        $model = new \app\admin\model\weblog\Log;
        $total = $model->where($where)->count('id');

        $data = [];
        if (!$total) {
            return false;
        }
        $data['site_id'] = $site_id;
        $data['visits_count'] = $total;
        $data['ip_count'] = $model->where($where)->group('ip')->count('id');

        $model = new \app\admin\model\weblog\SiteStatistics();
        $model->save($data);
        return true;
    }

    /**
     * ip统计
     * @param 类型 $site_id 站点id
     * @return bool 返回类型
     * @author Created by Xing <464401240@qq.com>
     */
    private function ipStatistics($site_id, $pid)
    {
        $this->output->writeln('Start statistical ip data...');
        $ipStatisticsModel = new \app\admin\model\weblog\IpStatistics();
        $model = new \app\admin\model\weblog\Log();
        $where['site_id'] = $site_id;
        $where['pid']  = $pid;

        $min_id = $model->group('ip')->where($where)->order('id', 'ASC')->value('id');
        $total  = $model->group('ip')->where($where)->order('id', 'ASC')->count('id');
        $pageTotal = ceil($total / $this->limit);
        for ($p = 0; $p < $pageTotal; $p++) {
            $offset = $p * $this->limit;
            $last_max_id = $min_id + $offset;

            $list = $model->group('ip')
                ->where($where)
                ->where(['id' => ['egt', $last_max_id]])
                ->field('ip,spider_id,count(ip) as visits_count')
                ->limit($offset, $this->limit)
                ->order('id', 'ASC')
                ->select();

            $data = [];
            foreach ($list as $key => $row) {
                $data[] = [
                    'pid' => $pid,
                    'site_id' => $site_id,
                    'spider_id' => $row->spider_id,
                    'ip' => $row->ip,
                    'visits_count' => $row->visits_count,
                ];
            }
            if ($data) {
                $ipStatisticsModel->saveAll($data);
            }
            $this->output->writeln("IP #{$pid} A total of {$total} data, completed {$offset}, {$this->limit}... <info>ok</info>");
        }
        return true;
    }

    /**
     * url统计
     * @param 类型 $site_id 站点id
     * @return bool 返回类型
     * @author Created by Xing <464401240@qq.com>
     */
    private function urlStatistics($site_id, $pid)
    {
        $this->output->writeln('Start statistical url data...');
        $urlStatisticsModel = new \app\admin\model\weblog\UrlStatistics();
        $model = new \app\admin\model\weblog\Log();
        $where['site_id'] = $site_id;
        $where['pid']  = $pid;

        $min_id = $model->group('url_code')->where($where)->order('id', 'ASC')->value('id');
        $total  = $model->group('url_code')->where($where)->order('id', 'ASC')->count('id');
        $pageTotal = ceil($total / $this->limit);
        for ($p = 0; $p < $pageTotal; $p++) {
            $offset = $p * $this->limit;
            $last_max_id = $min_id + $offset;

            $list = $model->group('url_code')
                ->where($where)
                ->where(['id' => ['egt', $last_max_id]])
                ->field('url,url_code,count(url_code) as visits_count')
                ->limit($offset, $this->limit)
                ->order('id', 'ASC')
                ->select();

            $data = [];
            foreach ($list as $key => $row) {
                $where['url_code'] = $row->url_code;
                $data[] = [
                    'pid' => $pid,
                    'site_id' => $site_id,
                    'url' => $row->url,
                    'visits_count' => $row->visits_count,
                    'ip_count' => $model->where($where)->group('ip')->count('id'),
                ];
            }
            if ($data) {
                $urlStatisticsModel->saveAll($data);
            }
            $this->output->writeln("URL #{$pid} A total of {$total} data, completed {$offset}, {$this->limit}... <info>ok</info>");
        }
        return true;
    }

    /**
     * 得到文件大小
     * @author Created by Xing <464401240@qq.com>
     */
    private function getSize($file)
    {
        $filesize = filesize($file);
        if ($filesize >= 1073741824) {
            $filesize = round($filesize / 1073741824 * 100) / 100 . ' gb';
        } elseif ($filesize >= 1048576) {
            $filesize = round($filesize / 1048576 * 100) / 100 . ' mb';
        } elseif ($filesize >= 1024) {
            $filesize = round($filesize / 1024 * 100) / 100 . ' kb';
        } else {
            $filesize = $filesize . ' b';
        }
        return $filesize;
    }
}
